# Scrapy settings for hmss project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/topics/settings.html
#

BOT_NAME = 'hmss'

SPIDER_MODULES = ['hmss.spiders']
NEWSPIDER_MODULE = 'hmss.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/6.0 (Windows NT 6.2; WOW64; rv:16.0.1) Gecko/20121011 Firefox/16.0.1'

# REVISIT: enable below if needed 

#RETRY_HTTP_CODES = [500, 502, 503, 504, 400, 408, 111]

#DOWNLOADER_MIDDLEWARES = {                          
#    'c4.downloadermiddleware.retry.ProxyRetryMiddleWare': 500,
#    'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': None,
#}

# TODO: consider enabling these once get familiar with their function
#DOWNLOAD_DELAY = 0.25
#DUPEFILTER=True
#COOKIES_ENABLED=False
#RANDOMIZE_DOWNLOAD_DELAY=True
#SCHEDULER_ORDER='BFO'
